package catchData;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.log4j.Logger;

/**
 * @author yun
 *
 */
public class Catch4Region {
	
	private static Logger LOG = Logger.getLogger(Catch4Region.class);
	private static HttpClient client;
	private static final String LOGIN_PAGE = "http://www.stats.gov.cn/tjbz/cxfldm/2012/";
	private static final String HOST = "www.stats.gov.cn";
	private static final String regex="<td.*?>.*?<a href='(.*?)'>(.*?)</a>";
	private static final String regex1="<td>.*?<a href='(.*?)'>(.*?)</a>.*?</td>.*?<td><a.*?>(.*?)</a>";
	
	private void initHttpClient() {
		if (client == null) {
			MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
			client = new HttpClient(connectionManager);
			client.getParams().setCookiePolicy(
					CookiePolicy.BROWSER_COMPATIBILITY); // 配置Cookie策略为浏览器默认策略
			client.getParams().setParameter(
					"http.protocol.single-cookie-header", true);
			// client.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET,
			// "UTF-8");
			client.getParams().setParameter(
					HttpMethodParams.HTTP_CONTENT_CHARSET, "GBK");
			client.getHttpConnectionManager().getParams().setConnectionTimeout(
					300000);
			client.getHttpConnectionManager().getParams().setSoTimeout(300000);
		}
	}
	
	private void setHeaders(HttpMethod method) {
		method.getParams().setParameter("http.protocol.cookie-policy",
				CookiePolicy.BROWSER_COMPATIBILITY);
		method.setRequestHeader("accept",
						"image/jpeg, application/x-ms-application, image/gif, application/xaml+xml, image/pjpeg, application/x-ms-xbap, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*");
		method.setRequestHeader("referer", LOGIN_PAGE);
		method.setRequestHeader("accept-encoding",
				"UTF-8,GBK,GB2312,gzip,deflate");
		method.setRequestHeader("accept-language", "zh-CN");
		method.setRequestHeader("content-type",
				"application/x-www-form-urlencoded");
		method.setRequestHeader(
						"user-agent",
						"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; staticlogin:product=cboxf2010&act=login&info=ZmlsZW5hbWU9UG93ZXJXb3JkMjAxME94Zl9VbHRpbWF0ZS5leGUmbWFjPUY4RDgxQUY2MTgwQjQ3REQ5MUI5QkM4NjU3MzAwRkU3JnBhc3Nwb3J0PSZ2ZXJzaW9uPTIwMTAuNi4zLjYuMiZjcmFzaHR5cGU9MQ==&verify=b226e2d473bb8dbbb37568db14fee0e4; staticlogin:product=cboxf2010&act=login&info=ZmlsZW5hbWU9UG93ZXJXb3JkMjAxME94Zl9TcGVjaWFsLmV4ZSZtYWM9RjhEODFBRjYxODBCNDdERDkxQjlCQzg2NTczMDBGRTcmcGFzc3BvcnQ9JnZlcnNpb249MjAxMC42LjMuNS4yJmNyYXNodHlwZT0x&verify=94c65611a34cfb4964638346b072d76e; staticlogin:product=cboxf09&act=login&info=ZmlsZW5hbWU9UG93ZXJ3b3JkMjAwOU94Zi4yNTI2OS40MDExLmV4ZSZtYWM9RjhEODFBRjYxODBCNDdERDkxQjlCQzg2NTczMDBGRTcmcGFzc3BvcnQ9JnZlcnNpb249MjAwOS4wNS4yNS4zLjI3MiZjcmFzaHR5cGU9MQ==&verify=32e210faa8ce04f90dfc4225f98e80ac; aff-kingsoft-ciba; TheWorld)");
		method.setRequestHeader("host", HOST);
		method.setRequestHeader("connection", "Keep-Alive");
		method.setRequestHeader("cache-control", "no-cache");
	}
	
	private void saveDataFile(String sFolder, String data, long lid) {
		saveDataFile(sFolder, data, lid + "");
	}

	private void saveDataFile(String sFolder, String data, String sid) {
		File folder = new File(sFolder);
		if (!folder.exists()) folder.mkdirs();
		PrintWriter out = null;
		String saveFile = sFolder + "/" + sid + ".htm";
		try {
			out = new PrintWriter(new OutputStreamWriter(new FileOutputStream(
					saveFile), "GBK"));
			out.println(data);
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			try {
				if (out != null) {
					out.close();
					out = null;
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		LOG.info("It got the file " + saveFile.replaceAll("\\\\", "/"));
	}
	
	public static void main(String[] args) {
		Catch4Region instance = new Catch4Region();
		instance.initHttpClient();
		PostMethod first = new PostMethod(LOGIN_PAGE);
		instance.setHeaders(first);
		Pattern p1 = Pattern.compile(regex);
		StringBuffer index = new StringBuffer();
		List<String> list1 = new ArrayList<String>();
		Set<String> set2 = new LinkedHashSet<String>();
		try {
			client.executeMethod(first);
			index.append(first.getResponseBodyAsString());
			Matcher m1 = p1.matcher(index);
			first.releaseConnection();
			while (m1.find()){
				list1.add(m1.group(1));
//				LOG.info(m1.group(1) + "," + m1.group(2) + "\t");
			}
			index.delete(0, index.length());
//			LOG.info(index);
//			PostMethod second = null;
			instance.setHeaders(first);
			Pattern p2 = Pattern.compile(regex1);
			for(int i = 0; i < list1.size(); i++) {
//				LOG.info(LOGIN_PAGE + list1.get(i));
				first = new PostMethod(LOGIN_PAGE + list1.get(i));
				instance.setHeaders(first);
				client.executeMethod(first);
				index.append(first.getResponseBodyAsString());
				m1 = p2.matcher(index);
				first.releaseConnection();
//				list1.clear();
//				LOG.info(m1.find());
				while (m1.find()){
//					System.out.println("/" + m1.group(1));
//					list2.add("/" + m1.group(1));
					set2.add(m1.group(1));
//					System.out.println(m1.group(2) + "," + m1.group(3));
				}
			}
			list1.clear();
			index.delete(0, index.length());
//			LOG.info(set2.size());
//			for(int j = 0; j < 2; j++) {
			Iterator<String> it2 = set2.iterator();
				while(it2.hasNext()) {
//					LOG.info(LOGIN_PAGE + it2.next());wrong
					first = new PostMethod(LOGIN_PAGE + it2.next());
					instance.setHeaders(first);
					client.executeMethod(first);
					index.delete(0, index.length());
					index.append(first.getResponseBodyAsString());
					m1 = p2.matcher(index);
					first.releaseConnection();
	//				list1.clear();
	//				LOG.info(m1.find());
					while (m1.find()){
	//					System.out.println("/" + m1.group(1));
	//					list1.add("/" + m1.group(1));
						System.out.println(m1.group(2) + "," + m1.group(3));
					}
			}
			
		} catch (HttpException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

}
